package com.green.cosme.batch.scraper;

import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;

import net.htmlparser.jericho.Config;
import net.htmlparser.jericho.Element;
import net.htmlparser.jericho.HTMLElementName;
import net.htmlparser.jericho.LoggerProvider;
import net.htmlparser.jericho.Source;

import org.apache.commons.lang.StringUtils;

public class KenkoComScraper {

	private static final int DEV_MODE_DEVELOP = 10;
	private static final int DEV_MODE_PRODUCT = 20;
	private int COUNT = 0;

	private static int DEV_MODE = DEV_MODE_DEVELOP;

	public static void main(String[] args) {

		DEV_MODE = DEV_MODE_DEVELOP;
		DEV_MODE = DEV_MODE_PRODUCT;

		// コンソールのログ出力を隠す。
		Config.LoggerProvider = LoggerProvider.DISABLED;

		List<Item> itemList = null;
		KenkoComScraper scraper = new KenkoComScraper();
		try {
			itemList = scraper.exec();
		} catch (Exception e) {
			e.printStackTrace();
		}

		// for (Item item : itemList)
		// System.out.println(item);
	}

	public List<Item> exec() throws Exception {

		List<Item> tmpItemList = new ArrayList<Item>();

		// TOPページからカテゴリーまとめページを取得します。
		List<PageInfo> categoryList = getCategoryGroupList();

		for (PageInfo categoryPageInfo : categoryList) {

			System.out.println("カテゴリ：" + categoryPageInfo.getTitle());

			// カテゴリーまとめページからカテゴリページリストを取得します。
			List<PageInfo> seibunList = getSeibunList(categoryPageInfo);

			System.out.println("\t成分:" + seibunList.size() + "件数");
			// System.out.println("Category");
			// for (PageInfo categoryInfo : categoryList)
			// System.out.println(categoryInfo);

			// for (PageInfo categoryPageInfo : categoryList) {
			//
			// // 成分ページからアイテム一覧を取得します。
			// List<PageInfo> seibunList = getSeibunList(categoryPageInfo);

			// System.out.println("Seibun");
			// for (PageInfo seibunInfo : seibunList)
			// System.out.println(seibunInfo);

			for (PageInfo seibunInfo : seibunList) {

				System.out.println("成分：" + seibunInfo.getTitle());

				// if(!seibunInfo.getTitle().equals("クリーム"))
				// continue;

				List<PageInfo> seibunPageList = getSeibunPageList(seibunInfo);

				System.out.println("\t" + seibunPageList.size() + "ページ");

				for (PageInfo seibunPageInfo : seibunPageList) {

					List<PageInfo> itemPageInfoList = getItemPageInfoList(seibunPageInfo);

					System.out.println("\tアイテム:" + itemPageInfoList.size()
							+ "件");

					List<Item> tmpList = new ArrayList<Item>();

					for (PageInfo itemPageInfo : itemPageInfoList) {

						Item item = getItem(itemPageInfo, categoryPageInfo,
								seibunPageInfo);

						tmpItemList.add(item);

						tmpList.add(item);
					}

					for (Item tmpItem : tmpList)
						System.out.println(tmpItem);

					System.out.println();
				}
			}
			// }
		}

		return tmpItemList;
	}

	private List<PageInfo> getItemPageInfoList(PageInfo seibunPageInfo)
			throws Exception {

		URL seibunUrl = new URL(seibunPageInfo.getUrl());

		List<PageInfo> itemPageList = new ArrayList<PageInfo>();
		Source source = new Source(seibunUrl);

		List<Element> linkElements = source.getAllElements(HTMLElementName.A);
		for (Element linkElement : linkElements) {
			String href = linkElement.getAttributeValue("href");
			if (href == null)
				continue;
			String label = linkElement.getContent().getTextExtractor()
					.toString();

			// カテゴリーに関するリンクのみ取得します。
			if (href.contains("product/item/itm_")) {
				if (!href.contains("http://www.kenko.com")
						&& !href.contains("http://sg.kenko.com"))
					href = "http://www.kenko.com" + href;
				itemPageList.add(new PageInfo(label, href));
			}
		}

		if (isDevMode()) {
			System.out.println("DEV Mode: itemList:" + itemPageList.size());
			itemPageList.clear();
			if (COUNT == 0)
				itemPageList
						.add(new PageInfo("ランコム ジェニフィック 【海外処方品】 50ml",
								"http://localhost:8888/design/scraping/05_itm_6919854772.html"));
			else if (COUNT == 1)
				itemPageList
						.add(new PageInfo("ミラヴィ リッチエッセンス 30mlの海外発送",
								"http://localhost:8888/design/scraping/05_itm_6919911472.html"));
			COUNT++;
		}

		return itemPageList;

	}

	private List<PageInfo> getSeibunPageList(PageInfo seibunInfo)
			throws Exception {

		String strUrl = seibunInfo.getUrl();
		URL url = new URL(strUrl);

		List<PageInfo> seibunPageList = new ArrayList<PageInfo>();
		seibunPageList.add(seibunInfo);
		Source source = new Source(url);

		String[] tmp = strUrl.split("/");
		String tmp2 = tmp[tmp.length - 1];
		tmp2 = tmp2.split("\\.")[0] + "_";
		if (isDevMode())
			tmp2 = tmp2.replaceAll("04_", "");

		List<Element> linkElements = source.getAllElements(HTMLElementName.A);
		for (Element linkElement : linkElements) {
			String href = linkElement.getAttributeValue("href");
			if (href == null)
				continue;
			String label = linkElement.getContent().getTextExtractor()
					.toString();

			// 成分に関するリンクのみ取得します。
			if (href.contains(tmp2)) {
				if (!href.contains("http://www.kenko.com/"))
					href = "http://www.kenko.com" + href;
				String[] tmp10 = href.split(tmp2);
				String tmp12 = tmp10[1].split("\\.")[0];
				if (StringUtils.isNumeric(tmp12))
					addPage(seibunPageList, new PageInfo(label, href));
			}
		}

		if (isDevMode()) {
			System.out.println("DEV Mode: Category:" + seibunPageList.size());
			seibunPageList.clear();
			seibunPageList
					.add(new PageInfo("保湿美容液",
							"http://localhost:8888/design/scraping/04_sei_662016.html"));
			seibunPageList
					.add(new PageInfo("2",
							"http://localhost:8888/design/scraping/04_sei_662016_02.html"));
		}

		return seibunPageList;

	}

	private Item getItem(PageInfo itemPageInfo, PageInfo categoryPageInfo,
			PageInfo seibunPageInfo) throws IOException {

		// アイテムページからアイテム情報を取得します。
		URL itemUrl = new URL(itemPageInfo.getUrl());

		// 関連カテゴリー一覧
		List<PageInfo> categoryList = new ArrayList<PageInfo>();
		categoryList.add(categoryPageInfo);
		List<PageInfo> seibunList = new ArrayList<PageInfo>();
		seibunList.add(seibunPageInfo);
		String name = null;
		String brand = null;
		String maker = null;
		Integer price = null;
		Integer kComPrice = null;
		String janCode = null;
		String amount = null;

		Source source = new Source(itemUrl);
		List<Element> elementList = source.getAllElements();
		for (Element element : elementList) {

			String text = element.getContent().getTextExtractor().toString();

			String itemPropValue = element.getAttributeValue("itemprop");
			if ("category".equals(itemPropValue)) {
				List<Element> linkElements = element
						.getAllElements(HTMLElementName.A);
				for (Element linkElement : linkElements) {
					String href = linkElement.getAttributeValue("href");
					if (href.contains("cat/cat_")
							|| href.contains("seibun/sei_"))
						categoryList.add(new PageInfo(text, href));
				}
			} else if ("name".equals(itemPropValue)) {
				name = text;
				// } else if ("brand".equals(itemPropValue)) {
				// brand = text;
			} else if ("price".equals(itemPropValue)) {
				String str = extractNumber(text);
				price = Integer.parseInt(str);
			} else if ("identifier".equals(itemPropValue)) {
				janCode = text.replace("JANコード：", "").trim();
			}

			List<Element> tableElements = source
					.getAllElements(HTMLElementName.TABLE);
			for (Element tableElement : tableElements) {
				if ("mainTable".equals(tableElement.getAttributeValue("class"))) {
					List<Element> tdElements = tableElement
							.getAllElements(HTMLElementName.TD);
					for (Element tdElement : tdElements) {
						String text2 = tdElement.getContent()
								.getTextExtractor().toString();
						if (text2.contains("当社販売価格：")) {

							List<Element> divElements = tdElement
									.getAllElements(HTMLElementName.DIV);
							for (Element divElement : divElements) {
								String sss = divElement.getContent()
										.getTextExtractor().toString();
								if (sss.contains("ブランド：") && sss.split("ブランド：").length>=2)
									brand = sss.split("ブランド：")[1];
								String[] ttt = sss.split(" ");
								if ("販売元".equals(ttt[0]))
									maker = ttt[1];

								String[] text3 = text2.split(" ");

								for (String text4 : text3) {
									if (text4.contains("メーカー希望小売価格：") && text4
											.split("メーカー希望小売価格：").length>=2) {
										String strPrice = text4
												.split("メーカー希望小売価格：")[1];
										strPrice = extractNumber(strPrice);
										price = Integer.parseInt(strPrice
												.replace(",", "").replace("\\",
														""));
									} else if (text4.contains("当社販売価格：") && text4
											.split("当社販売価格：").length>=2) {
										String strKComPrice = text4
												.split("当社販売価格：")[1];
										strKComPrice = extractNumber(strKComPrice);
										kComPrice = Integer
												.parseInt(strKComPrice);
									}
								}
							}
						}
					}
				}
			}
		}

		List<Element> divElements = source.getAllElements(HTMLElementName.DIV);
		for (Element divElement : divElements) {
			String value = divElement.getContent().getTextExtractor()
					.toString();
			if (value.startsWith("内容量：") && value.split("内容量：").length >= 2)

				amount = value.split("内容量：")[1];
		}

		// return new Item(categoryCode, seibunCode, name, brand, maker,
		// kComPrice, amount, janCode);

		// List<Item> itemList = new ArrayList<Item>();
		// // for (PageInfo category : categoryList)
		// // for (PageInfo seibun : seibunList) {
		// itemList.add(new Item(categoryPageInfo.getTitle(),
		// seibunPageInfo.getTitle(),
		// name, brand, maker, price, kComPrice, amount, janCode));
		// // }
		//
		// return itemList;

		return new Item(categoryPageInfo.getTitle(), seibunPageInfo.getTitle(),
				name, brand, maker, price, kComPrice, amount, janCode);
	}

	private String extractNumber(String strKComPrice) {

		String strNumber = "";
		for (int i = 0; i < strKComPrice.length(); i++) {
			char c = strKComPrice.charAt(i);
			if (StringUtils.isNumeric("" + c))
				strNumber += c;
		}

		return strNumber;
	}

	private List<PageInfo> getSeibunList(PageInfo categoryPageInfo)
			throws Exception {

		URL categoryUrl = new URL(categoryPageInfo.getUrl());

		List<PageInfo> seibunList = new ArrayList<PageInfo>();
		Source source = new Source(categoryUrl);

		List<Element> linkElements = source.getAllElements(HTMLElementName.A);
		for (Element linkElement : linkElements) {
			String href = linkElement.getAttributeValue("href");
			if (href == null)
				continue;
			String label = linkElement.getContent().getTextExtractor()
					.toString();

			// カテゴリーに関するリンクのみ取得します。
			if (href.contains("product/seibun/sei_")) {
				if (!href.contains("http://www.kenko.com/"))
					href = "http://www.kenko.com" + href;

				seibunList.add(new PageInfo(label, href));
			}
		}

		if (isDevMode()) {
			System.out.println("DEV Mode: seibunList:" + seibunList.size());
			seibunList.clear();
			seibunList
					.add(new PageInfo("保湿美容液",
							"http://localhost:8888/design/scraping/04_sei_662016.html"));
		}

		return seibunList;
	}

	private void addPage(List<PageInfo> pageInfoList, PageInfo newPageInfo) {

		boolean isExist = false;

		for (PageInfo pageInfo : pageInfoList) {
			if (pageInfo.getUrl().equals(newPageInfo.getUrl())) {
				isExist = true;
				break;
			}
		}

		if (!isExist)
			pageInfoList.add(newPageInfo);
	}

	private List<PageInfo> getCategoryGroupList() throws Exception {

		URL url = null;
		if (isDevMode())
			// From File(Test)
			url = new URL(
					"http://localhost:8888/design/scraping/01_sei_all_cosme.html");
		else
			// From Web
			url = new URL("http://www.kenko.com/sei_all_cosme.html");

		List<PageInfo> categoryGroupList = new ArrayList<PageInfo>();
		Source source = new Source(url);

		List<Element> linkElements = source.getAllElements(HTMLElementName.A);
		for (Element linkElement : linkElements) {
			String href = linkElement.getAttributeValue("href");
			if (href == null)
				continue;
			String label = linkElement.getContent().getTextExtractor()
					.toString();

			// カテゴリーに関するリンクのみ取得します。
			if (href.contains("product/cat/cat_")) {
				if (!href.contains("http://www.kenko.com/"))
					href = "http://www.kenko.com" + href;
				categoryGroupList.add(new PageInfo(label, href));
			}
		}

		if (isDevMode()) {
			System.out.println("DEV Mode: CategoryGroup:"
					+ categoryGroupList.size());
			categoryGroupList.clear();
			categoryGroupList
					.add(new PageInfo("美容液・オイル",
							"http://localhost:8888/design/scraping/02_cat_060618.html"));
		}

		return categoryGroupList;
	}

	private List<PageInfo> getCategoryList(PageInfo categoryGroupPageInfo)
			throws Exception {

		URL url = new URL(categoryGroupPageInfo.getUrl());

		List<PageInfo> categoryList = new ArrayList<PageInfo>();
		Source source = new Source(url);

		List<Element> linkElements = source.getAllElements(HTMLElementName.A);
		for (Element linkElement : linkElements) {
			String href = linkElement.getAttributeValue("href");
			if (href == null)
				continue;
			String label = linkElement.getContent().getTextExtractor()
					.toString();

			// 成分に関するリンクのみ取得します。
			if (href.contains("http://www.kenko.com/product/seibun/sei_"))
				categoryList.add(new PageInfo(label, href));
		}

		if (isDevMode()) {
			System.out.println("DEV Mode: Category:" + categoryList.size());
			categoryList.clear();
			categoryList
					.add(new PageInfo("保湿美容液",
							"http://localhost:8888/design/scraping/03_cat_060618061801.html"));
		}

		return categoryList;
	}

	private static boolean isDevMode() {

		return DEV_MODE == DEV_MODE_DEVELOP;
	}
}
